package proc

import (
	"github.com/PuerkitoBio/goquery"
	"github.com/hu17889/go_spider/core/common/page"
	corerequest "github.com/hu17889/go_spider/core/common/request"
	"github.com/hu17889/go_spider/core/pipeline"
	"github.com/hu17889/go_spider/core/spider"
	"net/http"
	"strings"
	"tesou.io/platform/poem-parent/poem-api/common/base"
	"tesou.io/platform/poem-parent/poem-api/module/core/enums"
	"tesou.io/platform/poem-parent/poem-api/module/core/pojo"
	"tesou.io/platform/poem-parent/poem-core/module/core/service"
)

type KDProcesser struct {
	KDService    service.KDService
	SpiderUrl string
	cookies []*http.Cookie
}

func GetKDProcesser() *KDProcesser {
	return &KDProcesser{}
}

func (this *KDProcesser) Startup() {
	newSpider := spider.NewSpider(this, "KDProcesser")

	//newSpider = newSpider.AddUrl(this.SpiderUrl, "html")
	base.Log.Info("start URL:,", this.SpiderUrl)
	hder := make(http.Header)
	hder.Add("User-Agent", `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36`)
	hder.Add("Upgrade-Insecure-Requests", `1`)

	req := corerequest.NewRequest(this.SpiderUrl, "html", "KDindex", "GET", "", hder, nil, nil,nil)
	newSpider.AddRequest(req)

	newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole())
	newSpider.SetThreadnum(1).Run()
}

func (this *KDProcesser) Process(p *page.Page) {
	request := p.GetRequest()
	if !p.IsSucc() {
		base.Log.Info("URL:,", request.Url, p.Errormsg())
		return
	}
	//var allData []map[string]interface{}
	zhi_list_slice := make([]interface{}, 0)
	zhi_list_update_slice := make([]interface{}, 0)
	level := enums.GetListLevel("凯迪")
	p.GetHtmlParser().Find(".contentMes").Each(func(i int, selection *goquery.Selection) {
		s := selection.Find("a").First()
		url, boolUrl := s.Attr("href")
		text  := s.Text()
		img := selection.Find("img").First()
		img_src, _ := img.Attr("src")
		if len(img_src) < 5{
			img_src, _ = img.Attr("data-original")
		}

		content_src := strings.TrimSpace(selection.Find(".indexs").First().Text())
		author_src := selection.Find("strong").First().Text()
		if len(url) != 0 {
			if boolUrl {
				zhi := new(pojo.List)
				zhi.ListId = strings.Replace(url, "https://www.9kd.com/", "", -1)
				zhi.Image = img_src
				zhi.Author = author_src
				zhi.Content = content_src
				zhi.Title =  text
				zhi.SUrl = url
				if len(zhi.Title) > 0 {
					//设置朝代
					zhi.Type = level
					exist := this.KDService.Exist(zhi)
					if !exist {
						zhi_list_slice = append(zhi_list_slice, zhi)
					} else {
						zhi_list_update_slice = append(zhi_list_update_slice, zhi)
					}
				}
			}
		}
	})
	//执行入库
	this.KDService.SaveList(zhi_list_slice)
	this.KDService.ModifyList(zhi_list_update_slice)

}

func (this *KDProcesser) Finish() {
	base.Log.Info("zhihu表抓取解析完成 \r\n")

}

